import scrapy
from ..items import CbcItem
from selenium import webdriver
from selenium.webdriver import FirefoxOptions
from threading import Thread, Lock
class ChinaSpider(scrapy.Spider):
    name = 'ctv'
    allowed_domains = ['https://www.ctvnews.ca/']
    start_urls = ['https://www.ctvnews.ca/search-results/search-ctv-news-7.137?page=0&sortOrder=date&q=China&fdate=one_year&ftype=ctvnews.StandardArticle&fpage=']
    count = 18
    def __init__(self):
        super(ChinaSpider, self).__init__(name='ctv')
        option = FirefoxOptions()
        option.headless = True
        self.driver = webdriver.Firefox(options=option)
#356
    def parse(self, response):
        tr_list = response.xpath("//div[@class='search-main']/ul/li")
        for i in range(2):
            tr = tr_list[i]
            item = CbcItem()
            href = tr.xpath("./h3/a/@href").extract_first()
            item['herf'] = href
            item['publish_date'] = tr.xpath("normalize-space(./div[@class='meta']/text())").extract_first()
            b =tr.xpath("./h3/a")
            item["title"] = b[0].xpath("normalize-space(string(.))").extract_first()
            item["brief"] = ""
            yield scrapy.Request(
                item['herf'],
                callback=self.parse_detail,
                meta={"item": item},
                dont_filter=True
            )
        self.count = self.count + 1
        if (self.count <= 356):
            print(self.count)
            yield scrapy.Request(
                "https://www.ctvnews.ca/search-results/search-ctv-news-7.137?page="+str(self.count)+"&sortOrder=date&q=China&fdate=one_year&ftype=ctvnews.StandardArticle&fpage=",
                callback=self.parse,
                dont_filter=True
            )
    def parse_detail(self, response):
        item = response.meta['item']
        item["content"] = response.xpath("//div[@class='clearfix']/div/p/text()").extract()
        item["content"] = "".join(item["content"]).replace('\t', '').replace('\n', '').replace('\\','')
        yield(item)